from scrapy.spider import Spider
from scrapy.selector import Selector

from spider.items import SpiderItem
import MySQLdb

class BitcarshopSpider(Spider):

    name = 'bitcarshop'

    conn = MySQLdb.connect(host="localhost",user="spider",passwd="Xjk@4047218?",db="spider")
    cursor = conn.cursor() 
    sql = "select url,pages from sp_ba_cstemp order by id asc"
    result = cursor.execute(sql)
    targetUrl = []

    for row in cursor.fetchall():
        if row[1] == 0:
            targetUrl.append(row[0])
        else:
            for i in range(row[1]):
                j = i + 1
                targetUrl.append(row[0]+"?page="+str(j))
#    print len(targetUrl)
#    print len(listUrl)
#    start_urls = ["http://dealer.bitauto.com/baiyin/volkswagen/"]
#    start_urls = ["http://dealer.bitauto.com/beijing/volkswagen/"]
    start_urls = targetUrl

    def parse(self,response):
       sel = Selector(response)
       areas = sel.xpath('//div[@id="divDealerList"]//div[@class="d_search_list"]')
       items = []
       for area in areas:
           item = SpiderItem()
           item["carshop"] = area.xpath('.//h3[1]//a/@title').extract()
           item["carshopurl"] = area.xpath('.//h3[1]//a/@href').extract()
           item["tel"] = area.xpath('.//ul//li[3]//text() | .//ul//li[3]//span/text()').extract()
           item["carshopb"] = area.xpath('.//ul//li[1]/@title').extract()
           item["address"] = area.xpath('.//ul//li[2]/@title').extract()
           item["type"] = "bitcarshop"
           items.append(item)
       return items





